package coms6111.astbbw.dataminer;

import java.io.*;
import java.util.regex.*;
import java.util.*;

public class NaiveGlimpseMine extends DataMine {
	public final static String INDEXER = "/home/gravano/Bin/glimpseindex";
	public final static String QUERY = "/home/gravano/Bin/glimpse";
	private final static String QUERY_ARGS = "-ciwNygs"; // or something like that
	private final static Pattern QUERY_REGEX = Pattern.compile("(\\d+)");

	protected String tmpdirPath = null;
	protected TreeSet<String> terms = new TreeSet<String>();
	protected ArrayList<String> termsToDelete = new  ArrayList<String>();

	public NaiveGlimpseMine(String datasetName) {
		super(datasetName);
	}

	@Override
	public void buildIndex() {
		logger.debug("Entering buildIndex()");
		File tmpdir = new File("/tmp/astbbw-datamining-tmp-" + name);
		File datadir = new File(DataMine.DATA_ROOT, name);
		tmpdir.mkdir();
		tmpdirPath = tmpdir.getPath();
		this.singletonSets = new TreeSet<ItemSet>();

		String cmdline[] = {INDEXER, "-H", tmpdirPath, "-o", datadir.getPath()};
		try {
			logger.debug("Calling " + INDEXER + " to generate glimpse index");
			Process p = Runtime.getRuntime().exec(cmdline);
			// finish the process
			// read total indexed files from output, or get it elsewhere
			if ( 0 != p.waitFor()) {
				logger.warn("Indexer exited with error");
				BufferedReader errors = new BufferedReader(new InputStreamReader(p.getErrorStream()));
				String t = null;
				while (null != (t = errors.readLine())) logger.debug(t);
				System.exit(2);
			}
			File fileNameFile = new File(tmpdir, ".glimpse_filenames");
			BufferedReader fileNameReader = new BufferedReader(new FileReader(fileNameFile));
			try{
				this.totalDocs=Integer.parseInt(fileNameReader.readLine());
			}
			catch(Exception e){
				e.printStackTrace();
			}
			logger.debug("Indexer seems to have indexed " + totalDocs + " files");


			File indexFile = new File(tmpdir, ".glimpse_index");
			logger.debug("Parsing terms out of index file");

			Pattern pat = Pattern.compile("[^a-z]");
			BufferedReader indexReader = new BufferedReader(new FileReader(indexFile));

			String line;
			int lineno = 0;
			while ( null != (line = indexReader.readLine())) {
				String[]test =null;
				++lineno;
				try{
					test= pat.split(line);
					if(test.length<=0 || (test[0]==null||test[0].equals(""))) {
						logger.warn("Found null or empty word on line " + lineno);
						continue;
					}
					//System.out.println(test[0]);
				}
				catch(Exception e){
					logger.error("Trapped exception on line " + lineno,e);
					continue;
				}
				//String term = line.substring(0, line.indexOf(2));
				terms.add(test[0]);

			}
			logger.debug("Found " + terms.size() + " unique terms in index");
			createDFStructure();
		} catch (FileNotFoundException e) {
			logger.fatal("Index file could not be read", e);
			throw new RuntimeException(e);
		} catch (IOException e) {
			logger.fatal("I/O error reading index file", e);
			throw new RuntimeException(e);
		} catch (InterruptedException e) {
			logger.fatal("Interrupted while waiting for glimpse or glimpsindex", e);
			throw new RuntimeException(e);
		}


	}

	@Override
	public double findSupport(String[] terms) {
		BufferedReader in;
		StringBuffer catenated = new StringBuffer(); // XXX join terms with space
		for (int i = 0; i < terms.length; i++) {
			if (i > 0) catenated.append(' ');
			catenated.append(terms[i]);
		}
		/*
		 * Build our own index?  For this one, just let glimpse handle it:
		 * glimpse -H news_idx -ciwyNg 'windows macintosh linux'
		 * (1 document)
		 *
		 */
		int start, end, count=0;
		String tword;
		Pattern pattern = NaiveGlimpseMine.QUERY_REGEX;
		Matcher match=null;

		String cmdline[] = {QUERY, "-H", tmpdirPath, QUERY_ARGS, catenated.toString() };


		try {
			Process p = Runtime.getRuntime().exec(cmdline);
			// XXX read the results into int
			count = 0;
			p.waitFor();
			in= new BufferedReader(
					new InputStreamReader(
							p.getErrorStream()));

			while ((tword= in.readLine()) != null) {
				match=pattern.matcher(tword);

				if(match.find()){
					start = match.start();
					end =match.end();
					count=Integer.parseInt(tword.substring(start, end));
				}
				else{
					logger.debug("Error from findSupport "+tword);
				}
			}
			in.close();
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
		return (double) count/totalDocs;
	}

	@Override
	protected Iterator<String> getTerms() {
		return terms.iterator();
	}


	protected void createDFStructure(){
		logger.debug("Entering createDFStructure()");
		Iterator<String> it =getTerms();
		Thread indexers[] = new Thread[4];
		for (int i = 0; i < indexers.length; i++) {
			Thread t =  new Thread( new TermIndexer(it));
			indexers[i] = t;
			t.start();
		}
		for (int i = 0; i < indexers.length; i++) {
			try {
				indexers[i].join();
			} catch (InterruptedException e) {
				// This should really never happen
				logger.fatal("InterruptedException waiting for indexer " + i,e);
				throw new RuntimeException(e);
			}
		}

		terms.removeAll(termsToDelete);

	}


	/**
	 * @param word
	 * @param pattern
	 * @return
	 */
	protected void indexTerm(String word) {
		int start;
		int end;
		int count;
		String tword;
		BufferedReader in;
		Matcher match;
		String cmdline[] = {QUERY, "-H", tmpdirPath, QUERY_ARGS, word };
		if (null == word || word.equals("")) return;
		Process p = null;
		try {
			p = Runtime.getRuntime().exec(cmdline);
			// XXX read the results into int
			count = 0;
			p.waitFor();
			in= new BufferedReader(
					new InputStreamReader(
							p.getErrorStream()));

			while ((( tword= in.readLine())) != null) {
				match = QUERY_REGEX.matcher(tword);

				if(match.find()){
					start = match.start();
					end =match.end();
					count=Integer.parseInt(tword.substring(start, end));
				}
				else{
					logger.debug("Error from createDFStructure "+tword);
				}


			}

			in.close();
			synchronized (singletonSets) {
				singletonSets.add(
						new ItemSet(word, (double)count/this.totalDocs)
				);
			}
			//System.out.println(word+" "+count);

		} catch (Exception e) {
			throw new RuntimeException(e);
		} finally {
			try {
				p.getErrorStream().close();
				p.getInputStream().close();
				p.getOutputStream().close();
				p.destroy();
			} catch (Exception e) {
				logger.error("Error finalizing glimpse process for " + word,e);
			}
		}
	}

	public void findCommon(int n){
		// TODO Auto-generated method stub
		// find the top i terms, print them out, and remove them from the terms set
		// optionally, use this step to start the a-priori itemset-building process
		//System.out.println("Removing top "+n +" words");
		logger.debug("findCommmon called to remove top "  + n + " terms");
		ArrayList<ItemSet> temp = new ArrayList<ItemSet>();
		ArrayList<String> commonWords = new ArrayList<String>();
		ItemSet curr=null;
		for(Iterator<ItemSet> i= singletonSets.iterator();i.hasNext();){
			curr=i.next();
			//System.out.println(curr);
			temp.add(curr);
			String words[] = curr.getItems();
			// assertion: all of these are single-item sets
			commonWords.add(words[0]);
			n--;
			if(n==0){
				this.maxSupport = curr.getSupport();
				break;
			}
		}
		printToFileList(commonWords, name);
		remove(temp);
	}

	//removes terms in temp from the two sets
	private void remove(ArrayList<ItemSet> temp) {
		// TODO Auto-generated method stub
		ItemSet curr = null;
		for(Iterator<ItemSet> i= temp.iterator();i.hasNext();){
			curr =i.next();
			String tmp[] = curr.getItems();
			singletonSets.remove(curr);
			terms.remove(tmp[0]);
		}
		printToFileTreeSet(terms, name);
	}

	private void printToFileList(ArrayList<String> list, String name){
		Collections.sort(list);
		FileWriter fstream =null;
		BufferedWriter out;
		String curr;
		try{
			fstream = new FileWriter("COMMON-"+list.size()+"-"+name+".txt");
			System.out.println("The COMMON words list is written into - COMMON-"+list.size()+"-"+name+".txt");
			out = new BufferedWriter(fstream);
			for(Iterator<String> i= list.iterator();i.hasNext();){
				curr =i.next();
				out.write(curr);
				out.newLine();
			}
			out.close();
		}
		catch(IOException e){
			logger.debug("Error in opening output files");
			return;
		}


	}

	private void printToFileTreeSet(TreeSet<String> list, String name){

		FileWriter fstream =null;
		BufferedWriter out;
		String curr;
		try{
			fstream = new FileWriter("WORDS-"+list.size()+"-"+name+".txt");
			System.out.println("The WORDS list is written into - WORDS-"+list.size()+"-"+name+".txt");
			out = new BufferedWriter(fstream);
			for(Iterator<String> i= list.iterator();i.hasNext();){
				curr =i.next();
				out.write(curr);
				out.newLine();
			}
			out.close();
		}
		catch(IOException e){
			logger.debug("Error in opening output files");
			return;
		}


	}

	public void destroy() {
		try {
			if (null != this.tmpdirPath)
				Runtime.getRuntime().exec("rm -rf " + this.tmpdirPath).waitFor();
			tmpdirPath = null;
		} catch (Exception e) {
			logger.error("Exception in cleanup", e);
		}
		super.destroy();
	}

	protected class TermIndexer implements Runnable {

		private Iterator<String> it;

		public TermIndexer(Iterator<String> i) { it = i; }

		public void run() {
			NaiveGlimpseMine m = NaiveGlimpseMine.this;
			int indexcount = 0;
			m.logger.debug("Starting run() for " + this);
			while(true) {
				String term;
				synchronized(it) {
					if (!it.hasNext()) break;
					term = it.next();
					m.indexTerm(term);
					indexcount++;
				}

			}
			m.logger.debug("Thread exiting: indexed count is " + indexcount);
		}
	}


}
